{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# K-fold cross validation - Regression Model\n",
    "Based on the [Ludwig regression example](https://ludwig-ai.github.io/ludwig-docs/examples/#simple-regression-fuel-efficiency-prediction)  \n",
    "\n",
    "[Data set](https://archive.ics.uci.edu/ml/datasets/auto+mpg)\n",
    "\n",
    "This example demonstrates teh following:\n",
    "\n",
    "- Download a data set and create a pandas dataframe\n",
    "- Create a training and hold-out test data sets\n",
    "- Create a Ludwig config data structure from the pandas dataframe\n",
    "- Run a 5-fold cross validation analysis with the training data\n",
    "- Use Ludwig APIs to train and assess model performance on hold-out test data set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import logging\n",
    "import os\n",
    "import os.path\n",
    "import shutil\n",
    "import tempfile\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import requests\n",
    "import scipy.stats as stats\n",
    "import seaborn as sns\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "from ludwig.api import kfold_cross_validate, LudwigModel"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Contstants"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "DATA_SET_URL = 'http://archive.ics.uci.edu/ml/machine-learning-databases/auto-mpg/auto-mpg.data'\n",
    "DATA_SET = 'auto_mpg.data'\n",
    "RESULTS_DIR = 'results'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Clean out previous results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "if os.path.isfile(DATA_SET):\n",
    "    os.remove(DATA_SET)\n",
    "    \n",
    "shutil.rmtree(RESULTS_DIR, ignore_errors=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Retrieve data from UCI Machine Learning Repository"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Download required data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "r = requests.get(DATA_SET_URL)\n",
    "if r.status_code == 200:\n",
    "    with open(DATA_SET,'w') as f:\n",
    "        f.write(r.content.decode(\"utf-8\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create Pandas DataFrame from downloaded data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(398, 8)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "raw_df = pd.read_csv(DATA_SET,\n",
    "                     header=None,\n",
    "                      na_values = \"?\", comment='\\t',\n",
    "                      sep=\" \", skipinitialspace=True)\n",
    "\n",
    "\n",
    "raw_df.columns = ['MPG','Cylinders','Displacement','Horsepower','Weight',\n",
    "                'Acceleration', 'ModelYear', 'Origin']\n",
    "raw_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>MPG</th>\n",
       "      <th>Cylinders</th>\n",
       "      <th>Displacement</th>\n",
       "      <th>Horsepower</th>\n",
       "      <th>Weight</th>\n",
       "      <th>Acceleration</th>\n",
       "      <th>ModelYear</th>\n",
       "      <th>Origin</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>18.0</td>\n",
       "      <td>8</td>\n",
       "      <td>307.0</td>\n",
       "      <td>130.0</td>\n",
       "      <td>3504.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>70</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>15.0</td>\n",
       "      <td>8</td>\n",
       "      <td>350.0</td>\n",
       "      <td>165.0</td>\n",
       "      <td>3693.0</td>\n",
       "      <td>11.5</td>\n",
       "      <td>70</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>18.0</td>\n",
       "      <td>8</td>\n",
       "      <td>318.0</td>\n",
       "      <td>150.0</td>\n",
       "      <td>3436.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>70</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>16.0</td>\n",
       "      <td>8</td>\n",
       "      <td>304.0</td>\n",
       "      <td>150.0</td>\n",
       "      <td>3433.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>70</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>17.0</td>\n",
       "      <td>8</td>\n",
       "      <td>302.0</td>\n",
       "      <td>140.0</td>\n",
       "      <td>3449.0</td>\n",
       "      <td>10.5</td>\n",
       "      <td>70</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    MPG  Cylinders  Displacement  Horsepower  Weight  Acceleration  ModelYear  \\\n",
       "0  18.0          8         307.0       130.0  3504.0          12.0         70   \n",
       "1  15.0          8         350.0       165.0  3693.0          11.5         70   \n",
       "2  18.0          8         318.0       150.0  3436.0          11.0         70   \n",
       "3  16.0          8         304.0       150.0  3433.0          12.0         70   \n",
       "4  17.0          8         302.0       140.0  3449.0          10.5         70   \n",
       "\n",
       "   Origin  \n",
       "0       1  \n",
       "1       1  \n",
       "2       1  \n",
       "3       1  \n",
       "4       1  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "raw_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create train/test split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(318, 8)\n",
      "(80, 8)\n"
     ]
    }
   ],
   "source": [
    "train_df, test_df = train_test_split(raw_df, train_size=0.8, random_state=17)\n",
    "print(train_df.shape)\n",
    "print(test_df.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Setup Ludwig config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_features = ['Cylinders', 'Displacement', 'Horsepower', 'Weight', 'Acceleration', 'ModelYear']\n",
    "cat_features = ['Origin']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create Ludwig input_features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_features = []\n",
    "# setup input features for numerical variables\n",
    "for p in num_features:\n",
    "    a_feature = {'name': p, 'type': 'numerical', \n",
    "                'preprocessing': {'missing_value_strategy': 'fill_with_mean', 'normalization': 'zscore'}}\n",
    "    input_features.append(a_feature)\n",
    "\n",
    "# setkup input features for categorical variables\n",
    "for p in cat_features:\n",
    "    a_feature = {'name': p, 'type': 'category'}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create Ludwig output features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "output_features =[\n",
    "    {\n",
    "        'name': 'MPG',\n",
    "        'type': 'numerical',\n",
    "        'num_fc_layers': 2,\n",
    "        'fc_size': 64\n",
    "    }\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'input_features': [{'name': 'Cylinders',\n",
       "   'type': 'numerical',\n",
       "   'preprocessing': {'missing_value_strategy': 'fill_with_mean',\n",
       "    'normalization': 'zscore'}},\n",
       "  {'name': 'Displacement',\n",
       "   'type': 'numerical',\n",
       "   'preprocessing': {'missing_value_strategy': 'fill_with_mean',\n",
       "    'normalization': 'zscore'}},\n",
       "  {'name': 'Horsepower',\n",
       "   'type': 'numerical',\n",
       "   'preprocessing': {'missing_value_strategy': 'fill_with_mean',\n",
       "    'normalization': 'zscore'}},\n",
       "  {'name': 'Weight',\n",
       "   'type': 'numerical',\n",
       "   'preprocessing': {'missing_value_strategy': 'fill_with_mean',\n",
       "    'normalization': 'zscore'}},\n",
       "  {'name': 'Acceleration',\n",
       "   'type': 'numerical',\n",
       "   'preprocessing': {'missing_value_strategy': 'fill_with_mean',\n",
       "    'normalization': 'zscore'}},\n",
       "  {'name': 'ModelYear',\n",
       "   'type': 'numerical',\n",
       "   'preprocessing': {'missing_value_strategy': 'fill_with_mean',\n",
       "    'normalization': 'zscore'}}],\n",
       " 'output_features': [{'name': 'MPG',\n",
       "   'type': 'numerical',\n",
       "   'num_fc_layers': 2,\n",
       "   'fc_size': 64}],\n",
       " 'training': {'epochs': 100, 'batch_size': 32}}"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "config = {\n",
    "    'input_features' : input_features,\n",
    "    'output_features': output_features,\n",
    "    'training' :{\n",
    "        'epochs': 100,\n",
    "        'batch_size': 32\n",
    "    }\n",
    "}\n",
    "config"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Perform K-fold Cross Validation analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "starting 5-fold cross validation\n",
      "training on fold 1\n",
      "CPU times: user 40.7 s, sys: 5.38 s, total: 46 s\n",
      "Wall time: 40.6 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "with tempfile.TemporaryDirectory() as tmpdir:\n",
    "    data_csv_fp = os.path.join(tmpdir,'train.csv')\n",
    "    train_df.to_csv(data_csv_fp, index=False)\n",
    "\n",
    "    (\n",
    "        kfold_cv_stats, \n",
    "        kfold_split_indices \n",
    "    ) = kfold_cross_validate(\n",
    "        num_folds=5,\n",
    "        config=config,\n",
    "        dataset=data_csv_fp,\n",
    "        output_directory=tmpdir,\n",
    "        logging_level=logging.ERROR\n",
    "    )\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'loss_mean': 8.111681,\n",
       " 'loss_std': 2.4598064,\n",
       " 'error_mean': 0.0380627,\n",
       " 'error_std': 0.5965346,\n",
       " 'mean_squared_error_mean': 8.111682,\n",
       " 'mean_squared_error_std': 2.4598064,\n",
       " 'mean_absolute_error_mean': 2.0598435,\n",
       " 'mean_absolute_error_std': 0.2779836,\n",
       " 'r2_mean': 0.8666786,\n",
       " 'r2_std': 0.03552912}"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "kfold_cv_stats['overall']['MPG']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train model and assess model performance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = LudwigModel(\n",
    "    config=config,\n",
    "    logging_level=logging.ERROR\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 8.34 s, sys: 1.78 s, total: 10.1 s\n",
      "Wall time: 15 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "training_stats = model.train(\n",
    "    training_set=train_df,\n",
    "    output_directory=RESULTS_DIR,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/project/ludwig/data/preprocessing.py:1045: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  computed_fill_value,\n"
     ]
    }
   ],
   "source": [
    "test_stats, mpg_hat_df, _ = model.evaluate(dataset=test_df, collect_predictions=True, collect_overall_stats=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'MPG': {'loss': 8.303831,\n",
       "  'error': -0.45136052,\n",
       "  'mean_squared_error': 8.303831,\n",
       "  'mean_absolute_error': 2.2274728,\n",
       "  'r2': 0.8558148},\n",
       " 'combined': {'loss': 8.303831}}"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/seaborn/_decorators.py:43: FutureWarning: Pass the following variables as keyword args: x, y. From version 0.12, the only valid positional argument will be `data`, and passing other arguments without an explicit keyword will result in an error or misinterpretation.\n",
      "  FutureWarning\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQ8AAAEKCAYAAAAM4tCNAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO2deXyV5Zn3v1cCkSVAIAkQNlnCBFFWEVCsgzqtYq1axe21o2Ndal9nxplWW+lMO52ZzttW/LR2fK0Oii86dbdYLVQZi4CliIgoO1ES1kDIQkLIRrbr/eM853hycnK25OzX9/PJ55znfrbrwHN+576v+7quW1QVwzCMcMmItwGGYSQnJh6GYUSEiYdhGBFh4mEYRkSYeBiGEREmHoZhRESfaF5cRA4Cp4F2oE1V54jIMOAVYDxwELhJVWuiaYdhGL1PLHoel6rqTFWd42w/DKxV1cnAWmfbMIwkIx7DlmuB55z3zwHXxcEGwzB6iEQzwlREDgA1gAL/parLRKRWVXOc/QLUuLd9zr0XuBdg4MCB50+ZMiVqdhpGutLWrpRW1XP66GdVqpofzrlR9XkAF6tqmYgMB94VkX3eO1VVRcSveqnqMmAZwJw5c3Tr1q1RNtUw0ouKumZueXozLaea2fvviw6Fe35Uhy2qWua8VgBvAHOBEyJSAOC8VkTTBsMwuuIWjvJTzay4c25E14iaeIjIQBEZ5H4PfAXYBbwF3OEcdgfwZrRsMAyjK77CMXfCsIiuE81hywjgDZdbgz7Ai6r6joh8BLwqIncBh4CbomiDYRhe9JZwQBTFQ1VLgRl+2quBy6N1X8Mw/NObwgEWYWoYaUFvCweYeBhGyhMN4QATD8NIaaIlHGDiYRgpSzSFA0w8DCMlibZwgImHYaQcsRAOMPEwjJQiVsIBJh6GkTLEUjjAxMMwUoJYCweYeBhG0hMP4QATD8NIauIlHGDiYRhJSzyFA0w8DCMpibdwgImHYSQdiSAcYOJhGElFoggHmHgYRtKQSMIBJh6GkRQkmnCAiYdhJDyJKBxg4mEYCU2iCgeYeBhGwpLIwgEmHoaRkCS6cICJh2EkHMkgHGDiYRgJRbIIB5h4GEbCkEzCASYehpEQJJtwgImHYcSdZBQOMPEwjLiSrMIBJh6GETeSWTjAxMMw4kKyCweYeBhGzEkF4QATD8OIKakiHGDiYRgxI5WEA0w8DCMmpJpwgImHYUSdVBQOMPEwjKiSqsIBMRAPEckUkU9EZJWzPUFEPhSR/SLyiohkRdsGw4gHqSwcEJuexwPAXq/tnwO/VNVCoAa4KwY2GEZMSXXhgCiLh4iMAb4KPONsC3AZ8LpzyHPAddG0wTBiTToIB0S/5/EY8D2gw9nOBWpVtc3ZPgqM9neiiNwrIltFZGtlZWWUzTSM3iFdhAOiKB4icjVQoaofR3K+qi5T1TmqOic/P7+XrTOM3iedhAOgTxSvvQC4RkSuAvoBg4FfATki0sfpfYwByqJog2HEhHQTDohiz0NVl6jqGFUdD9wCvKeqtwHrgMXOYXcAb0bLBsOIBekoHBCfOI/vA98Rkf24fCDL42CDYfSItnaXGy9dhQOiO2zxoKrrgfXO+1Jgbizuaxi9TWllPW/vKufP+6uYMTaHVduPUd3QknbCATESD8NIBUor67nxqQ+obmgBYFNJNQCP3Twj7YQDLDzdMELm7V3lHuHwpqy2OQ7WxB8TDyMpcPsY4nWvtvYO/ry/yu/xm0qqaI+hfYmCDVuMhMbbx7CgMI9F541kYn52zO/VJzODGWNz+OjgSTo6YPqYIeRmZ3GgqpGLJuWRmZl+v8MmHkbC4s/H8OzGA7x234W9LiDB7lVR18zvtx9DFc4pGMRFhblMLRhMcflpFhalZxBj+smlkTT48zFUN7Tw9q7ymN6roq6ZG57cxNGaJto6lF3H6nhiXQk/enM3BTn9uX35Fkor63vdpkTHxMNISHriYwjXPxLoXuuLK7hl2WbK67o6RasbWthfUU/OgKyoCFqiY+JhJCR9MjNYUJjnd193PobSynqeWLef25/dwhPr9ofcGwh0r5LKBsrrmikaMcjv/n3ldYzPG5CWTlMTDyNhWXTeSHIHdq4VlTswi0XnjexyrNtnsXRNMZtKqlm6ppgbn/ogZAHxd68MgcaWNlbcOZdF0wr8njdl5GAOpqnTNL0+rZFUTMzP5rX7LuShK4pYUJjLQ1cUdess7al/xPteF4wfyrCBWWT1yeD5b85j7oRh3QpZ4fBsahtb/ApaqiOqGm8bgjJnzhzdunVrvM0w4kh7e0e3v+xt7R3c/uwWT8SnNwsKc3n+zrkh9woq6po9Pg7fkHP3VO6mkipmjRvK1IJBHD/VzKVFw6M2fRwrRORjVZ0Tzjk2VWskBYG+/G6fhT/xCGc44Uly8yMc4Oqd3H9pIfddMpHMzIyAgpYOpO8nN1KKcPwj/mZjwsmOdQtGOgsHWM/DSBHcPgv3sOKiSV2jUbuLIE3ntPqeYD4PI+XwHk60tXfQJzOjSwQpuHomT31jNt9fuTPthcN8Hkba4y0Wm0urKattYnROf8YNG0CHzw9ldUML3/rNNppb29NaOCLFxMNICbyHJAuL8pkxJocDVQ3sPlZHc2sHWZkZ/PPVU/nuq9s7nVfb2MKLd88z4YiAgOIhIv8ZwjXqVPWfe8kewwgb3yHJ12aM4n+/sK1TklvuwCwevXEGk/KzKfEKHLv5grHMn+Q/utQITLCex7XAj4Ic8zBg4mHEDe8AscLh2eyvqPcbMPbhgWpmjB3iEY/B/fpwz5cmxtxe99Aq2QkmHr9U1ecCHSAiQ3vRHsMIC9+ktvG5A9h7vM7vsR8fquHIySYyxNXjuOdLE2Ma3BXL2iSxIKB4qOpjwS4QyjGGES18A8QOVDWysCjfb8DY/op6zrR18OLd82I+VIllbZJYEbDvJCLnisg1Xtu/FJFnnb/Z0TfPMILjHSBWUlnPlJGD/Ca5NTmzKvHwccSyNkmsCDbw+hngXejgCmA1roWbgvlCDCMm+CbQjR3anyWLpnDXxRO4YPxQhvTvS5+MDB65YVpcZlVStf5pMPEoUNVNXtt1qvpbVf1vwFzURkwJVOTHnXfy/J1z+fhwLQ++voM/7jnB7mN11DW10tLewZGa+FQ5j6Q2STIQzOpOFVBUdb7X5vDeN8cwuhJOkR8Fz6/8oZONNLa04w4Ni+evfDi5N8lCsNmWYyIyT1U/9G4UkfnAseiZZRguQnU0uqc/3VXOe5ph29uEknuTbAQTj+8Dr4jICmCb03Y+rgWqb46iXYYBfOFonJSfzYS8ARyoaqTEmfK8/9LCLtOfc8cPZdWOrr9rifAr75vSn+wEm6rdIiLzgL8F/sZp3g3MV9UTUbbNSHPOtLaz40gtP71+Gvsr6tl7vI6FRfnc/aUJrC+uoKTiNDf91+ZOvZIMgaw+GTx28wzKapsT8lc+FYQDQsttGQXsAF5S1b1RtscwPEltTa1t3H3JRO7774+7hJovu/183vz0WJfpzw6Fr88azXWzxgCkzK98IhIszuNHwKvADcBqEbknJlYZacvBqnq2HaqhpLKBtXsr2VBcyYNXFDF0QF/PMdUNLRyubmTroRoAzh42gAFZmYiz//DJRo9j1IQjegTredwMzFTVRhHJBd4Bno6+WUa6crSmiZ++va9LT+N7V05hycqdnuM2lVYxb0Iuf/kX+fx6fQnNre1cNa2Aiyfn0djSZqIRA4L9C59R1UYAVa0O4XjDiJi29g427q/yG4m5v6KeSV4+i8H9spgyMpufv7OPU02tdCis3nmcR9cUM39ibqxNT0uC9TwmishbznsBJnlto6rX+D/NMCJjx9FTftvdiyuVVNaTOzCLSfkD+d5vd9LhUwivuqGF9cWVnDtqSAysTW9CScn35tFoGWIYfTIzmD8x12+MxqyxQzlxuomHrihi+pgh3P/CNurPtPm9zqaSKnOUxoBgU7UbIr2wiPQD3gfOcu7zuqr+i4hMAF4GcoGPgb9W1Zbur2SkAqHWsLh6egHPbTrYpdbo9bNHc/awAVQ3tHDL05tpbe/gulmjWbmtrMs1kjnkO5kIVklsR6D9qjo9wO4zwGWqWi8ifYGNIvI28B1cdUJeFpGngLuAJ8O020gSwq1hESgS07vK+XPfnEdedhYbiiu7CE28g8HShYDV00XkU1zpAi8CvweavPer6qGQbiIyANgIfBtXVu5IVW0TkQuBH6vqFYHOt+rpyUl3FctDrWHhXQW9u+URymoa2fBZJat3Ho9qMFiqVP/qjl6vnq6qM0VkCnArLgHZ47z+j6r6H3B2NigT19CkEHgCKAFqvc49Cozu5tx7gXsBxo0bF9KHMRKLQDUs7r+0MOj5gYTDt0fzb9ecy6Th/ley7wmpVv2rNwlr3RYRuRmXCPxcVZeGcV4O8AbwQ2CFqhY67WOBt1X1vEDnW88j+eit9WO9hWP5HXO4cFJej3s0oRKr+yQCkfQ8gv7vichoEfmuiGwEvgH8I2H6KFS1FlcBoQuBHBFx93jGAF09XkbS0xs1LNzCcay2iWtnjuLx9/bz4oeHWLXjeEyqcqVi9a/eJFh4+gZcvo6+wJ24smlXA1kiErAkk4jkOz0ORKQ/8GVgLy4RWewcdgfwZk8+gJG49KSGhbdwZGVm8NKWI2wqqea9fRVsLu3am4HerdeRqtW/epNgcR5n43KYfgvH/+AgTnuguvUFwHOO3yMDeFVVV4nIHuBlEfkJ8AmwPFLjjcTAnzOxrb0j4hoW3kOVr88azUtbjnj2BSpw3JtTtL6FlaN1n2QmmMN0fKQXVtUdwCw/7aXA3EivayQO/pyJGQKrd3ZuC6eGha+P4/H39nfaX1JZz91fmkDuwKyoT9EuOm8kz248YFPB3RBsqnakqgYc4IVyTE8xh2ni0Z0zccmiKTz4+o5ObaE6GP3Nqjyxbj9L1xR3KgZ0suEMj986i+1HT0W9XodbIBOxLkhvEo2Frv8ABFtiIZRjjBSirb2DzaXVfp2Je8tPd1rSMdSp2e7iOL46bSQjBp3F3vLTnmJA54wcxJih/bl4cn7Uw9BTrfpXbxJMPGaIiP/lt1wIEGi/kUJ4D1OmjxnCT6+fxiPv7KOmsdVzjHcCm5tguSbdCQe4ivv4S9F/7b4LgdjV6zDh6Eown0dmrAwxEhO3M9RfIWJ/dTamjBzM+uLKTtcI5GAMJBzQ80AzI3qYnBp+8V7uIFBshXedjdyBWZwzclCnXkcgB2Mw4bDp0sQmlBqmRprh28sYkJVJY0u732P3ldex6LwR9M8a7ZlteeiKoqAOxmDCATZdmuiYeBhd8B0qBIutuO+SiSh4Yj2CORhDEQ43Nl2auIQkHiIyCTiqqmdEZCEwHXjeCTs3Ugh/Q4VAsRULi/J56v3SLolj3QUAhCMckJqLJaUKISXGOan5c4DxuKZm3wTOVdWromqdg8V5xBZ3bIU3Qwf07RJbsbAon79/8RNKqho8x+UOzOJXt8zk1+tLumShhiscvnin6Bu9S1QS4xw6nDT6rwOPq+pDuMLPjRTEX05Khgijcvp7FpO+/9JC1hdXdhIOcDlR1xVXcqLuDEvXFHPjUx9QWlkfknAEWsgabLo00QjV59EqIrfiSmT7mtPWN8DxRhITbKiQmZkRcCbEO9ajuqGF1z4+yprd5d0Kh9XMSE5CHbZMBe4DPlDVl5w6pDep6s+jbSDYsKW3iKQaVqChgr/hDcBdF09w9UqcKdv+fTMQkW6FI11qZiQyURu2qOoeVf17VX3J2T4QK+Eweo53zMYT6/ZT6hWHEQy3cPgbUnSXcl84PLtTrEdbh3Y7VLGaGclLqLMtC4Af40rR74OTkq+qgVLyjTjh3cPwFxn67MYDIf+yBxpS+A5v5k3IZdSQfvzHHzovabx08fRufRyBgsAsnySxCdXnsRxXBbGPAf/RQkbc8f2iXzOjoEfh3aEIj2/iWGllPbfMHceKPx+grUNZuni6Z9FpXywILLkJVTxOqerbUbXE6BH+vuifnzhNxekzfo8P5Zc9HOFxXyf7rD6s2V2OiPDC3fOCTsdaEFjyEqp4rBORpcBKXOuxAKCq26JilRE2/r7oO8vqWDRtZES/7OEMKdzDpEjiOCwILHkJVTzmOa/e3lgFLutdc4xI6O6LXlJZz9SCQRFV3QplSOE9TJoxNodV249R3dASdgCY1cxITkISD1W9NNqGGJET6It+/FRzxL/sgYYU/oZJAI/dPCPsyFE3JhzJRaizLUOAfwEucZo2AP+mqv6XNDdiTndf9EuLhgOu8PKZY3IYOqBrbF938R8T87N55VvzWbP7RBfheWLd/i7DJICy2uZe/FRGIhPqsOVZYBdwk7P918D/A66PhlFG+HTnO8gQuOFJ/0FYQLfTsN5DkqunF7D0humMGjoAsClWw0Wo4jFJVW/w2v5XJ1nOSCD8+Q789RCqG1pYteM4Gz6r5ONDNUDnaVjAb9Uw9xRtn8wMZozNsSnWNCfU/+UmEbnYveEEjTUFON6II96zIN31ED48UE2OzxCmuqGFzaXVQaM+K+qaWbX9WJdr2hRrehFqz+PbuBZwGoIruvQk8DfRMsroHQI5UqeNHsK7eyq6tB+rbWLbYf9lWjaVVHH9zFHc9uwWqhtaeOzmGZTVNtsUa5oS6mzLp7gqqQ92tq1iepLQnSP14sI8ntpQ2uX4UTn96Z/Vx6/gTB+Tw23PbukSx2E+jvQkoHiIyDdU9Tci8h2fdgBU9RdRtM3oBQI5Uv3Ff8yfmAvQRXCG9u/bbRyHP+GIJIPXSC6C9TwGOq+D/OwLnstvJATdBWEFiv/w3jd9TOAAMN9EPKvNkR6EWs9jgar+OVhbtLB6HtElUM2O4zWNfocq0FUouitLaLU5Ep9oliF8PMQ2IwkJVOU8kHDc+NQHLF1TzKaSapauKeb25Vu4+5LOVRqsNkfqEszncSFwEZDv4/cYDNhqcilMpCu5uReBCme5SSM5Cfa/mQVk4xKZQV5/dcDi6Jpm9JRgBYW7oycrubnrl3pjgWOpSbC1ajcAG0RkhaoeipFNRoR4ryvrz2kZygxIT1dym1owmPf2fbFWrQWOpS6hBok9IyI3uhd5EpGhwMuqekX0TDNCxVss5k90lQJ85k+l1DS2svd4HSMGncWqHcfZXFodcAakN1Zyu27WaHIGZFngWBoQ6mzLJ6o6K1hbtLDZlu7jJrqrPv7gFUUsWbmTn14/jUfXFAetTh5uIZ+DVfVsPVjD3vLT7CuvY8rIwZwzchBzxg9lfF62LdCUZER10ScRGed1o7OxOI+YEKzyeSDH5WVFw9lfUR+0OnkkFcBW7yznwdd3sL64kn59M1lfXMmDr+9g9U7XdU04Up9Q/4f/CdgoIv8tIr8B3geWBDpBRMaKyDoR2SMiu0XkAad9mIi8KyKfO69De/YRUhd/06HuFdgguONyxrgh7D3uP5NgU0kV7e0dEQmH931LKutZu7fCM7vivq6R+oS6bss7wGzgFeBl4HxVXRPktDbgu6o6FZgP3O8sHvUwsFZVJwNrnW3DD8GyW92OS39MGTmYTw+f4pyCwX73XzQpj+qGFo9wLL9jTlgVwK6e7n+1UZtZSR8C/i+LyBTndTYwDjjm/I1z2rpFVY+7CySr6mlgLzAauBZ4zjnsOeC6nnyAVCVYwR33r3t3Cy9dXJhHS3s7Fxfm+d0/d/xQbnl6M8dqm7h25igef29/0AWhvIdQVfUtPLp4eqfKZDazkl4EdJiKyNOqeo+IrPOzW1U1pALIIjIe11DnPOCwquY47QLUuLd9zrkXuBdg3Lhx5x86lH4zxd0t5/jQFUWdlj5wz7ZsKqli2ughnJ07kFe3HmHYwCxONrRw05yxHKpuYGfZKaaNHsK5owbzyJpiKk+fISszg7rmNs+1ugsn784x+6tbZvLkhhKbWUlyInGYhjTb0hNEJBtXzdP/UNWVIlLrLRYiUqOqAf0e6TrbEu46ru3tHby45TA/fHN3l30PLyriSHUTG/dXcehkIxkCi88fw6tbj3Y51lecILCQWfRo8hOJeAQLTw9Yo1RVVwY5vy/wW+AFr2NPiEiBqh4XkQKga0UaAwh/TRPFFaTlL9V+SP8sfrbliy//rHE5HK3xXwzOO5zcHaUarGapkX4ECxL7mvM6HFeOy3vO9qXAJlyLQPnFGZIsB/b61P14C7gD+Jnz+mb4ZqcPoaxp4rt+yhO3zWbj51V8cqSGaaNzmDZ6MD94Y6fn+OyzMvnmggnsOV7HppJqJuVnMyFvAAeqGimprOeiSXkcO9XEW9uPewogz5+YazVLjU4EC0+/E0BE/geYqqrHne0CYEWQay/AVWV9p1ex5B/gEo1XReQu4BBfVGQ3AhBIOHyLFb/60RF+ePVU9pbXUVx+mlNNLZxq+sKv8U9fncqP3tzND756Do8uns7e8tPsPV7HwqJ8vv2XEzlvzBCue2JTp2s+unh6RItHGalLqOHpY93C4XAC1+xLt6jqRlz1Tv1xeYj3NYLQ3XTuzrJTHHR6EvmDzvLsKxye7Qkc6+hQfvb2vi5V0h+/dVaXa/7HH/by+K2z2H70lIWeG0Do4rFWRNYALznbNwN/jI5JRqgECxL72oyRPP/BYU43t3qKFTe1tLHtcC2Fw7PZV37ar/C8/3lll7T6msZWntxQwvN3zjUHqQGEXgD5b0Xk63yxYtwyVX0jemYZoRAou3Vi3kB+s/kwjS1tPP/NL1arb2/v4Kn3S/nkcE230ac7y04xPm9AJ/EA828YnQnnSdgGrFbVfwTWiIi/uqZGjFlYlN8lCGxo/768s+sEVfUtZGVmkJf9xf7MzAwWnTeSmsbWbqNP503IpbaxtVOb+TcMX0Jdq/YeXAFbw4BJuCJFn8J8F1Ej1Orjm0urefCKIg5UNbD72CnGDRvAH/dUUFl/BoC65jbe3lXeKW5jYn42//BXk2lrV373SVkXJ+jV0wu4enpBRItjG+lDqD6P+4G5wIcAqvq5iAyPmlVpTDjVx9vaO1i7t4JNJdX89fxxTM7P5oUth2lt7xz451sGsKymkSfXl9DU2s5Pvn4enxyuZfexU0wZOZjzz87h7GEDyMzMCDpFbKQ3oYrHGVVtca/XIiJ9sJT8XsfftKt7/Vh/AuLt83j/sypqm1q7CAd09VVs+KyScwoGs3zjAb79m21Mys9mfN4A1hdXMmxgVqdjTTiM7gj1ydggIj8A+ovIl4HXgN9Hz6z0JFgWrT8WnTeSof37cuhkI00t7Qzu1/n3wNdX0dbewaodxykcnu3xlbjT6msbW7jy3BG9+ImMVCbUnsf3gbuBncC3gD8Az0TLqHQkWBat7/DB7RPJPqsP2f36UN/SxgN/Vch5o3P4oKSKnWWnmDdhGFdPH9Wp1+LurTzyzj6+d+UU9lfUeyqBXVqUz6Th5gc3QiOoeIhIJrBbVacAT0ffpPQk0LSr97DDNxTdvZLb0sXT+fdVe6luaPEMQzZ8VsXsca6cQ28BcdcfXbJyp+fYT4/Uctu8gHF/htGJoOKhqu0iUiwi41T1cCyMSle6KyrsHnb484kAngAwd3tJZb0nRmNdcSWfHqll6eLpHgHxTbibPW6ozaYYYRPqsGUosFtEtgCetQRV9ZqoWJWmBMuiXbXjONUNLZw9bACV9WdoamlHgcMnGzl8stHvNfeV15EzoK/f6VqbTTF6Qqji8cOoWmF46O5L3dbewe6yUyxZNIVfry+hubWdq6YVcPHkPNYXV3DD7DG8/nFZl+tNGTmY9cWVNLe2+xUKEw4jUoLV8+gH3AcU4nKWLlfVtkDnGL2Dvy/1VdNG8o+vbqfDmY1dvdO1FssvbppBzoAsv1mvhcOzWb7xANfPHm1CYfQqwXoezwGtwJ+ARcBU4IFoG5VuhBJNerKhhR//fo9HONxUN7TwyZFapo8ewhO3zWZzSTUfHTrJlJGDKRyezSPv7LPQciMqBBOPqao6DUBElgNbom9S+hBqNGlFXTM3L9vMqaZWP1eBjw6e5NqZo1j85AecnTuAv7tsMvvK61i14xh3XjSeq2eMMmeo0esEEw/P06qqbe4IU6PnhBpN+mFpFd9+4RNONbVy3azRrNzW1a9x0aQ8PiipprqhheqGFu5c8ZFnCjZv0FkmHEZUCDYIniEidc7faWC6+72I+M/nNkLCPXPiTXVDC6t2uGouldU08vbO49z69IecbGihvUO5YPwwv8soXHHuCM95btxRo6t3HrdFmIyoEKwMYWasDEkn2to72FzaNRgM4MMD1fxxzyB+9+kxNu6v6uTjcEeFltU28cnhmk5TuaEEmBlGb2JPVZyYPmaI3/Zpo4fwh13lvLevgjofH0dNYytLVu7kRF0T910yibOH9ffs627xJ3OUGtEi1DgPoxfpk5nBxYV5vLb1aJep1aIRg1i+8QBt7cpV0wpYvfN4l/OHD+rHm9vLGNI/i+bWDjIk/GUaDKOnmHjEiTFD+7Nk0RT2lp/2JKaNHtKP//P2Pk9a/cKifDaXVncRmDFDB/DEuhLP9q9umcn4vGyLGjViiolHDwm14pcv4/Oy6VDo1zeDqQWDaG5t5/+uK6Hy9BnPMRkZ0klgZo7NYczQATzyzj7PMdUNLWzcX8WFE3M9gmHCYcQCE48ICafily/egnPoZBPr9lVQWtVAY0sbo4b049ipZgqHZ7PnWB3LNx5gUn4218wo4EhNo6fH4c3OslO9+tkMIxRMPCIg3Ipf3ue5BWf+xFxGDenHsvdLPAsyDe7Xh6fvmMP64krPEgngmnZ9a/txFhbl+73uBeOHWW/DiDkmHhEQqOKX7wLRbvwJztD+fcnKzARc4lHX3MbavRX8/eWTPUskuKdfSyrruftLE/zmr0wZOYj29g4TECOm2NMWJsEqfnUXkOVPcGqaWqlpbGHUkC+mXD88UO0RAt/p10fe2ceSRVN44PJCFhTmcu8lE3nwiiLKaptMOIyYYz2PMAm14pc3gQRnSsEgRgzux7FTrhXrp4/J8ezznX6dNyEXgD99XsXQgVmeuqOv3Xdhb3w0wwgLE48ICFbxy5dAgjNvQi7riys91/jK1BGdBMh3+rW0sp4Tp5IjfPAAAApVSURBVM+wqaSK62ePtlgOI26IauKvoDBnzhzdunVrvM3ohNv5GWpAVmllPTf8ehM1XlGjuQOzePTGGTyzsdSTQn+6uZV7L5kU9P7m4zB6ExH5WFXnhHOO9TwiJNyALO8q50UjB3FxYR5n5w7kP9/7nGEDs1hfXMnyjQdYUJjLXQsmBL2mCYcRb0w8ekgoX+KKumZueXoz1Q0tvHD3fM4fl8MrW4+wZOXOLsdaIpuRLNhTGmXcwlF+qpkVd85l7gRXTMb8ibmWyGYkNdbziCL+hMONJbIZyY45TKNEIOHwxZyfRryJxGEatSdWRJ4VkQoR2eXVNkxE3hWRz53XodG6fzwJRzjAnJ9GchLNp3YFcKVP28PAWlWdDKx1tlOKcIXDMJKVqImHqr4PnPRpvhbXcg44r9dF6/7xwITDSCdi3V8eoaru0ljlwIjuDhSRe0Vkq4hsraysjI11PcCEw0g34jbYVpentltvraouU9U5qjonP99/KnqiYMJhpCOxFo8TIlIA4LxWxPj+vY4Jh5GuxFo83gLucN7fAbwZ4/v3KiYcRjoTzanal4APgCIROSoidwE/A74sIp8Df+VsJyUmHEa6E7UIU1W9tZtdl0frnrHChMMwLLclbEw4DMOFiUcYmHAYxheYeISICYdhdMbEIwRMOAyjKyYeQTDhMAz/mHgEwITDMLrHxKMbTDgMIzAmHn4w4TCM4Jh4+GDCYRihYeLhhQmHYYSOiYeDCYdhhIeJByYchhEJaS8eJhyGERlpLR4mHIYROWkrHiYchtEz0lI8TDgMo+eknXiYcBhG75BW4mHCYRi9R9qIhwmHYfQuaSEeJhyG0fukvHiYcBhGdEhp8TDhMIzokbLiYcJhGNElJcXDhMMwok/KiYcJh2HEhpQSDxMOw4gdKSMeJhyGEVtSQjxMOAwj9iS9eJhwGEZ8SGrxMOEwjPiRtOJhwmEY8SUpxcOEwzDiT9KJhwmHYSQGSSUeJhyGkTgkjXiYcBhGYhEX8RCRK0WkWET2i8jDwY5va1cTDsNIMGIuHiKSCTwBLAKmAreKyNRA55RW1ZtwGEaCEY+ex1xgv6qWqmoL8DJwbaATWtvVhMMwEow+cbjnaOCI1/ZRYJ7vQSJyL3Cvs3lm3sTcXTGwrTfIA6ribUQYJJO9yWQrJJe9ReGeEA/xCAlVXQYsAxCRrao6J84mhUQy2QrJZW8y2QrJZa+IbA33nHgMW8qAsV7bY5w2wzCSiHiIx0fAZBGZICJZwC3AW3GwwzCMHhDzYYuqtonI3wJrgEzgWVXdHeS0ZdG3rNdIJlshuexNJlshuewN21ZR1WgYYhhGipM0EaaGYSQWJh6GYUREQotHuGHssUZEnhWRChHZ5dU2TETeFZHPndeh8bTRjYiMFZF1IrJHRHaLyANOe6La209EtojIdsfef3XaJ4jIh84z8YrjdE8IRCRTRD4RkVXOdiLbelBEdorIp+5p2nCfhYQVj0jC2OPACuBKn7aHgbWqOhlY62wnAm3Ad1V1KjAfuN/590xUe88Al6nqDGAmcKWIzAd+DvxSVQuBGuCuONroywPAXq/tRLYV4FJVnekVixLes6CqCfkHXAis8dpeAiyJt11+7BwP7PLaLgYKnPcFQHG8bezG7jeBLyeDvcAAYBuuSOQqoI+/ZyTONo5xvnCXAasASVRbHXsOAnk+bWE9Cwnb88B/GPvoONkSDiNU9bjzvhwYEU9j/CEi44FZwIcksL3OMOBToAJ4FygBalW1zTkkkZ6Jx4DvAR3Odi6JayuAAv8jIh87qSAQ5rOQsOHpqYCqqogk1Fy4iGQDvwX+QVXrRMSzL9HsVdV2YKaI5ABvAFPibJJfRORqoEJVPxaRhfG2J0QuVtUyERkOvCsi+7x3hvIsJHLPI1nD2E+ISAGA81oRZ3s8iEhfXMLxgqqudJoT1l43qloLrMPV9c8REfePXqI8EwuAa0TkIK4s8cuAX5GYtgKgqmXOawUuYZ5LmM9CIotHsoaxvwXc4by/A5dvIe6Iq4uxHNirqr/w2pWo9uY7PQ5EpD8u/8xeXCKy2DksIexV1SWqOkZVx+N6Tt9T1dtIQFsBRGSgiAxyvwe+Auwi3Gch3o6bIE6dq4DPcI11/yne9vix7yXgONCKa0x7F66x7lrgc+CPwLB42+nYejGuce4O4FPn76oEtnc68Ilj7y7gR077RGALsB94DTgr3rb62L0QWJXItjp2bXf+dru/W+E+CxaebhhGRCTysMUwjATGxMMwjIgw8TAMIyJMPAzDiAgTD8MwIsLEwzCMiDDxSHBEJNdJm/5URMpFpMxru8cp3iLyLyLyU5+2mSKyN8A5PxaRB3t67wDXd6eLz3G214vIYfGKpReR34lIvfN+vIg0Of8me0TkKRHJcPZNFpFVIlLi5HGsE5FLnH03O+nyq6L1WVIZE48ER1Wr1ZU2PRN4CleK90znr8Ur/DlSXgJu9mm7xWmPJ5eqqvdyALW4wsBxIk8LfI4vcf6NpuMq4XCdiPQDVgPLVHWSqp4P/B2uIClU9RXg7uh+jNTFxCMJEZEVzq/rh8Ajvj0BEdnlZM4iIt9wiup8KiL/5dRJ8aCqnwE1IuK98NZNwEsico+IfOQU5PmtiAzwY8t6rx5CnpPf4c6IXeqcv0NEvuW0F4jI+449u0TkSyF+7JdxiRrA9cBKfwepK4t1E1AI3AZ8oKpvee3fpaorQrynEQATj+RlDHCRqn6nuwNE5BxcvYoFzq9yO64vlC8v4XwxnYI7J1X1c2Clql6groI8ewmvmM1dwClVvQC4ALhHRCYA/wtXXYuZwAxcYfKhsBa4xBG/W4BX/B3kCNzlwE7gXFx1QIwoYCn5yctr6kpZD8TlwPnAR467oD/+MyVfATaJyHfpPGQ5T0R+AuQA2biWywiVrwDTRcSdGDYEmIwr4fFZJ8P3d6oaqni0Axsd+/qr6kHvcgLAJKf2hwJvqurbIvJl7wNE5A3Hhs9U9fowPovhBxOP5KXB630bnXuR/ZxXAZ5T1SWBLqSqR0TkAPCXwA24Ut/BVWbxOlXdLiJ/gyvpyxfve/fzahfg71S1i+A4DsuvAitE5Beq+nwg+7x4GVf6+I/97HP7PLzZDVzi3lDVrztDrEdDvJ8RABu2pAYHgdkAIjIbmOC0rwUWOwVf3AVuz+7mGi8BvwRKVfWo0zYIOO70EvwNd9z3Pt95v9irfQ3wbedcROQvnFTws4ETqvo08Izb7hD5E/BTQnfmvggsEJFrvNq6+G2MyLCeR2rwW+B2EdmNq7TgZwCqukdE/hlXubkMXKUD7gcO+bnGa8B/4pqNcPND53qVzusgP+c9CrwqrlJ2q73an8FV33WbM8VaCVyHq/fykIi0AvXA7aF+SHWlgIfca1DVJnFV+fqFiDwGnABOAz8J9RpG91hKvpFwODM2c1S1Kgb3Wgg8qKpXR/teqYYNW4xEpBJY654CjhYicjPwa1zLIhhhYj0PwzAiwnoehmFEhImHYRgRYeJhGEZEmHgYhhER/x89paLFxrVANQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "a = plt.axes(aspect='equal')\n",
    "sns.scatterplot(test_df['MPG'].values, mpg_hat_df['MPG_predictions'].values,\n",
    "               s=50)\n",
    "plt.xlabel('True Values [MPG]')\n",
    "plt.ylabel('Predictions [MPG]')\n",
    "lims = [0, 50]\n",
    "plt.xlim(lims)\n",
    "plt.ylim(lims)\n",
    "_ = plt.plot(lims, lims)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Compare K-fold Cross Validation metrics against hold-out test metrics"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Hold-out Test Metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'loss': 8.303831,\n",
       " 'error': -0.45136052,\n",
       " 'mean_squared_error': 8.303831,\n",
       " 'mean_absolute_error': 2.2274728,\n",
       " 'r2': 0.8558148}"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_stats['MPG']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### K-fold Cross Validation Metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'loss_mean': 8.111681,\n",
       " 'loss_std': 2.4598064,\n",
       " 'error_mean': 0.0380627,\n",
       " 'error_std': 0.5965346,\n",
       " 'mean_squared_error_mean': 8.111682,\n",
       " 'mean_squared_error_std': 2.4598064,\n",
       " 'mean_absolute_error_mean': 2.0598435,\n",
       " 'mean_absolute_error_std': 0.2779836,\n",
       " 'r2_mean': 0.8666786,\n",
       " 'r2_std': 0.03552912}"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "kfold_cv_stats['overall']['MPG']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
